{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import codecs\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '1'\n",
    "gpus = tf.config.experimental.list_physical_devices(device_type='GPU')\n",
    "for gpu in gpus:\n",
    "    tf.config.experimental.set_memory_growth(gpu, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'batch_size': 64,\n",
    "    'lr' : 0.001,\n",
    "    'max_sent_len': 20,\n",
    "    'epochs': 50,\n",
    "    'drops' : [0.1]\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_data(data_path):\n",
    "    \"\"\"\n",
    "    意图识别抽取出label\n",
    "    槽位识别与填充作为命名实体识别问题，对每一个字进行实体标注, ate_time', 'B-target', 'I-date_time', 'I-date_time', 'I-operation', 'I-date_time', 'I-date_time']\n",
    "[ ]:\n",
    "￼\n",
    "​B E I O S\n",
    "    \"\"\"\n",
    "    with codecs.open(data_path,\"r\",encoding=\"utf-8\") as fp:\n",
    "        data = json.load(fp)\n",
    "    texts = [example['text'].replace(\" \",\"\") for example in data]\n",
    "    intent_labels = [example['intent'] for example in data]\n",
    "    \n",
    "    slots_ners = []\n",
    "    count = 0\n",
    "    for example in data:\n",
    "        if 'entities' in example.keys():\n",
    "            text = example['text']\n",
    "            ner = ['O'] * len(text)\n",
    "            slots = example['entities']\n",
    "            for key,val in slots.items():\n",
    "                start_idx = text.find(val)\n",
    "                end_idx = start_idx + len(val) -1\n",
    "                if len(val) == 1:\n",
    "                    ner[start_idx] = 'S-' + key\n",
    "                else:\n",
    "                    ner[start_idx] = 'B-' + key\n",
    "                    ner[end_idx] = 'E-'+ key\n",
    "                    for idx in range(start_idx+1, end_idx):\n",
    "                        ner[idx] = 'I-' + key\n",
    "        else:\n",
    "            text = example['text']\n",
    "            ner = ['O'] * len(text)\n",
    "        slots_ners.append(ner)\n",
    "    print('texts len: ', len(texts))\n",
    "    print('intent_lables len: ',len(intent_labels))\n",
    "    print('slots_ners len: ', len(slots_ners))\n",
    "    return texts, intent_labels, slots_ners        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "texts len:  2542\n",
      "intent_lables len:  2542\n",
      "slots_ners len:  2542\n"
     ]
    }
   ],
   "source": [
    "data_path =\"../dataset/data_v2.json\"\n",
    "max_sent_len = params[\"max_sent_len\"]\n",
    "texts, intent_labels, slots_ners = extract_data(data_path)\n",
    "# l = len(texts) // params['batch_size']\n",
    "# texts = texts[:l*params['batch_size']]\n",
    "# intent =  intent[:l*params['batch_size']]\n",
    "# slots_ners = slots_ners[:l*params['batch_size']]\n",
    "train_text = [d for i , d in enumerate(texts) if i % 10 != 0]\n",
    "train_l = len(train_text) // params['batch_size']\n",
    "train_text = train_text[:train_l*params['batch_size']]\n",
    "valid_text = [d for i , d in enumerate(texts) if i % 10 == 0]\n",
    "valid_l = len(valid_text) // params['batch_size']\n",
    "valid_text = valid_text[:valid_l*params['batch_size']]\n",
    "\n",
    "train_intent = [d for i , d in enumerate(intent_labels) if i % 10 != 0]\n",
    "train_intent = train_intent[:train_l*params['batch_size']]\n",
    "valid_intent = [d for i , d in enumerate(intent_labels) if i % 10 == 0]\n",
    "valid_intent = valid_intent[:valid_l*params['batch_size']]\n",
    "\n",
    "train_ner = [d for i , d in enumerate(slots_ners) if i % 10 != 0]\n",
    "train_ner = train_ner[:train_l*params['batch_size']]\n",
    "valid_ner = [d for i , d in enumerate(slots_ners) if i % 10 == 0]\n",
    "valid_ner =valid_ner[:valid_l*params['batch_size']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../char_6.17.json', mode='r', encoding='utf-8') as f:\n",
    "    dicts = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "char2id = dicts['char2id']\n",
    "id2char = dicts['id2char']\n",
    "intent2id = dicts['intent2id']\n",
    "id2intent = dicts['id2intent']\n",
    "slot2id = dicts['slot2id']\n",
    "id2slot = dicts['id2slot']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['intent_num'] = len(intent2id)\n",
    "params['slot_num'] = len(slot2id)\n",
    "params['id2intent'] = id2intent\n",
    "params['id2slot'] = id2slot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trans2labelid(vocab, labels, max_sent_len):\n",
    "    labels = [vocab[label] for label in labels]\n",
    "    if len(labels) < max_sent_len:\n",
    "        labels += [0] * (max_sent_len - len(labels))\n",
    "    else:\n",
    "        labels = labels[:max_sent_len]\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(txt_seqs, intent_labels, slot_ners,char2id,intent2id,slot2id,max_sent_len):\n",
    "    dataset_text_labels = []\n",
    "    dataset_intent_labels = []\n",
    "    dataset_ner_labels = []\n",
    "    \n",
    "    for index in range(len(txt_seqs)):\n",
    "        dataset_text_labels.append(trans2labelid(char2id,txt_seqs[index],max_sent_len))\n",
    "        dataset_intent_labels.append([intent2id[intent_labels[index]]])\n",
    "        dataset_ner_labels.append(trans2labelid(slot2id,slot_ners[index],max_sent_len))\n",
    "    dataset_text_labels = np.array(dataset_text_labels)\n",
    "    dataset_intent_labels = np.array(dataset_intent_labels)\n",
    "    dataset_ner_labels = np.array(dataset_ner_labels)\n",
    "    \n",
    "    return dataset_text_labels, dataset_intent_labels, dataset_ner_labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tarin_seq, train_intent, train_ner =  read_data(train_text, train_intent, train_ner,char2id,intent2id,slot2id,max_sent_len) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_seq, valid_intent, valid_ner =  read_data(valid_text, valid_intent, valid_ner,char2id,intent2id,slot2id,max_sent_len) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Dataset(txt_seqs, dataset_intent_labels, dataset_ner_labels):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(({\n",
    "    \"Input\" : txt_seqs\n",
    "    },\n",
    "    {\n",
    "        \"pre_intent\":dataset_intent_labels,\n",
    "        \n",
    "        \"pre_ner\":dataset_ner_labels\n",
    "    }))\n",
    "    dataset = dataset.batch(params['batch_size'])\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = Dataset(tarin_seq, train_intent, train_ner)\n",
    "valid_dataset = Dataset(valid_seq, valid_intent, valid_ner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Layer\n",
    "\n",
    "class MultiHeadAttention(Layer):\n",
    "    def __init__(\n",
    "            self,\n",
    "            heads,\n",
    "            head_size,\n",
    "            out_dim=None,\n",
    "            use_bias=True,\n",
    "#             max_value = 1,\n",
    "#             min_value = -1\n",
    "            **kwargs\n",
    "    ):\n",
    "        super(MultiHeadAttention, self).__init__(**kwargs)\n",
    "        self.heads = heads\n",
    "        self.head_size = head_size\n",
    "        self.out_dim = out_dim \n",
    "        self.use_bias = use_bias\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        super(MultiHeadAttention, self).build(input_shape)\n",
    "        self.q_dense = tf.keras.layers.Dense(\n",
    "            units=self.head_size * self.heads,\n",
    "            use_bias=self.use_bias,\n",
    "            kernel_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            bias_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            name = 'q'\n",
    "            \n",
    "        )\n",
    "        self.k_dense = tf.keras.layers.Dense(\n",
    "            units=self.head_size * self.heads,\n",
    "            use_bias=self.use_bias,\n",
    "            kernel_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            bias_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            name = 'k'\n",
    "        )\n",
    "        self.v_dense = tf.keras.layers.Dense(\n",
    "            units=self.head_size * self.heads,\n",
    "            use_bias=self.use_bias,\n",
    "            kernel_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            bias_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            name = 'v'\n",
    "        )\n",
    "        self.o_dense = tf.keras.layers.Dense( \n",
    "            units=self.out_dim,\n",
    "            use_bias=self.use_bias,\n",
    "            kernel_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            bias_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            name = 'o'\n",
    "        )\n",
    "\n",
    "    def call(self, inputs):\n",
    "        q = inputs\n",
    "        k = inputs\n",
    "        v = inputs\n",
    "        # 线性变化\n",
    "        qw = self.q_dense(q)\n",
    "        kw = self.k_dense(k)\n",
    "        vw = self.v_dense(v)\n",
    "        # 形状变换\n",
    "        qw = tf.reshape(qw, (-1, tf.shape(q)[1], self.heads, self.head_size))\n",
    "        kw = tf.reshape(kw, (-1, tf.shape(q)[1], self.heads, self.head_size))\n",
    "        vw = tf.reshape(vw, (-1, tf.shape(q)[1], self.heads, self.head_size))\n",
    "        # attention\n",
    "        qkv_inputs = [qw, kw, vw]\n",
    "        o = self.pay_attention_to(qkv_inputs)\n",
    "        o = tf.reshape(o, (-1, tf.shape(o)[1], self.head_size * self.heads))\n",
    "        o = self.o_dense(o)\n",
    "        return o\n",
    "\n",
    "    def pay_attention_to(self, inputs):\n",
    "        (qw, kw, vw) = inputs[:3]\n",
    "        a = tf.einsum('bjhd,bkhd->bhjk', qw, kw)\n",
    "        a = a / self.head_size ** 0.5\n",
    "        A = tf.nn.softmax(a)\n",
    "        o = tf.einsum('bhjk,bkhd -> bjhd', A, vw)\n",
    "#         print(o)\n",
    "        return o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import concatenate, Dropout,LayerNormalization, Dense, add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.models.Model):\n",
    "    def __init__(\n",
    "        self,\n",
    "        layer_count,\n",
    "        **kwargs\n",
    "    ):\n",
    "        super(Encoder, self).__init__(**kwargs)\n",
    "        self.layer_count = layer_count\n",
    "        \n",
    "    def build(self,input_shape):\n",
    "        self.MultiHeadAttention =  MultiHeadAttention(heads=16,head_size=4,out_dim=32)\n",
    "        self.dropout_1 = Dropout(0.1)\n",
    "        self.l1 =  LayerNormalization(name='normal')\n",
    "        self.feed1 = Dense(32,name='feed')\n",
    "        self.dropout1 = Dropout(0.1)\n",
    "        self.l_1 =  LayerNormalization(name='normal1')\n",
    "        \n",
    "    def call(self,inputs):\n",
    "        state = inputs\n",
    "        for i in range(self.layer_count):\n",
    "#             print('state: ',i)\n",
    "            att1 = self.MultiHeadAttention(state)\n",
    "            att_1 = add([att1,state])\n",
    "#             dropout1  = self.dropout_1(att_1)\n",
    "            l1 = self.l1(att_1)\n",
    "            feed1 =self.feed1(l1)\n",
    "#             dropout_1  = self.dropout1(feed1)\n",
    "            l_1 = self.l_1(feed1)\n",
    "            state = l_1\n",
    "        return state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input (InputLayer)              [(None, 20)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 20, 32)       16000       Input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "encoder (Encoder)               (None, 20, 32)       9600        embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d (Globa (None, 32)           0           encoder[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "pre_intent (Dense)              (None, 55)           1815        global_average_pooling1d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "pre_ner (Dense)                 (None, 20, 36)       1188        encoder[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 28,603\n",
      "Trainable params: 28,603\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "text_inputs = tf.keras.layers.Input(shape=(20,),name='Input')\n",
    "embed = tf.keras.layers.Embedding(500,32)(text_inputs)\n",
    "\n",
    "l_1 = Encoder(layer_count=1)(embed)\n",
    "\n",
    "conv = tf.keras.layers.GlobalAveragePooling1D()(l_1)\n",
    "pre_intent = tf.keras.layers.Dense(params['intent_num'],activation='sigmoid',name = 'pre_intent',kernel_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            bias_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0))(conv)\n",
    "pre_slot = tf.keras.layers.Dense(params['slot_num'],activation='sigmoid',name = 'pre_ner',kernel_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0),\n",
    "            bias_constraint = tf.keras.constraints.MinMaxNorm(min_value=-1.0))(l_1)\n",
    "model = tf.keras.Model(text_inputs,[pre_intent,pre_slot])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses = {'pre_intent':'sparse_categorical_crossentropy','pre_ner':'sparse_categorical_crossentropy'}\n",
    "metrics = { 'pre_intent': ['accuracy'],'pre_ner': ['accuracy']}\n",
    "optimizer = tf.keras.optimizers.Adam(params['lr'])\n",
    "model.compile(optimizer, loss=losses, metrics=metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath='../ner_model_weight/model_encoder_714.h5',save_weights_only=True,save_best_only=True)\n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(patience=20,factor=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "35/35 [==============================] - 0s 13ms/step - loss: 7.1468 - pre_intent_loss: 4.1929 - pre_ner_loss: 2.9539 - pre_intent_accuracy: 0.0067 - pre_ner_accuracy: 0.5575 - val_loss: 6.6937 - val_pre_intent_loss: 3.8435 - val_pre_ner_loss: 2.8502 - val_pre_intent_accuracy: 0.1719 - val_pre_ner_accuracy: 0.3531\n",
      "Epoch 2/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 6.3983 - pre_intent_loss: 3.8687 - pre_ner_loss: 2.5296 - pre_intent_accuracy: 0.0250 - pre_ner_accuracy: 0.6663 - val_loss: 5.9518 - val_pre_intent_loss: 3.7560 - val_pre_ner_loss: 2.1957 - val_pre_intent_accuracy: 0.1823 - val_pre_ner_accuracy: 0.7271\n",
      "Epoch 3/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 5.6860 - pre_intent_loss: 3.7199 - pre_ner_loss: 1.9661 - pre_intent_accuracy: 0.2348 - pre_ner_accuracy: 0.6810 - val_loss: 5.3284 - val_pre_intent_loss: 3.6561 - val_pre_ner_loss: 1.6723 - val_pre_intent_accuracy: 0.1615 - val_pre_ner_accuracy: 0.7138\n",
      "Epoch 4/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 5.1194 - pre_intent_loss: 3.5962 - pre_ner_loss: 1.5231 - pre_intent_accuracy: 0.2531 - pre_ner_accuracy: 0.6694 - val_loss: 4.8750 - val_pre_intent_loss: 3.5700 - val_pre_ner_loss: 1.3050 - val_pre_intent_accuracy: 0.2083 - val_pre_ner_accuracy: 0.7247\n",
      "Epoch 5/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 4.7320 - pre_intent_loss: 3.4932 - pre_ner_loss: 1.2388 - pre_intent_accuracy: 0.2817 - pre_ner_accuracy: 0.7242 - val_loss: 4.5850 - val_pre_intent_loss: 3.4905 - val_pre_ner_loss: 1.0945 - val_pre_intent_accuracy: 0.2292 - val_pre_ner_accuracy: 0.7557\n",
      "Epoch 6/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 4.4674 - pre_intent_loss: 3.4002 - pre_ner_loss: 1.0672 - pre_intent_accuracy: 0.2951 - pre_ner_accuracy: 0.7818 - val_loss: 4.3871 - val_pre_intent_loss: 3.4192 - val_pre_ner_loss: 0.9679 - val_pre_intent_accuracy: 0.2240 - val_pre_ner_accuracy: 0.7888\n",
      "Epoch 7/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 4.2732 - pre_intent_loss: 3.3153 - pre_ner_loss: 0.9579 - pre_intent_accuracy: 0.2973 - pre_ner_accuracy: 0.8152 - val_loss: 4.2463 - val_pre_intent_loss: 3.3538 - val_pre_ner_loss: 0.8925 - val_pre_intent_accuracy: 0.1823 - val_pre_ner_accuracy: 0.8128\n",
      "Epoch 8/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 4.1215 - pre_intent_loss: 3.2333 - pre_ner_loss: 0.8882 - pre_intent_accuracy: 0.2879 - pre_ner_accuracy: 0.8292 - val_loss: 4.1330 - val_pre_intent_loss: 3.2865 - val_pre_ner_loss: 0.8465 - val_pre_intent_accuracy: 0.1562 - val_pre_ner_accuracy: 0.8154\n",
      "Epoch 9/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 3.9909 - pre_intent_loss: 3.1492 - pre_ner_loss: 0.8418 - pre_intent_accuracy: 0.2710 - pre_ner_accuracy: 0.8350 - val_loss: 4.0248 - val_pre_intent_loss: 3.2105 - val_pre_ner_loss: 0.8143 - val_pre_intent_accuracy: 0.1562 - val_pre_ner_accuracy: 0.8242\n",
      "Epoch 10/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 3.8668 - pre_intent_loss: 3.0576 - pre_ner_loss: 0.8092 - pre_intent_accuracy: 0.2491 - pre_ner_accuracy: 0.8417 - val_loss: 3.9155 - val_pre_intent_loss: 3.1227 - val_pre_ner_loss: 0.7928 - val_pre_intent_accuracy: 0.1562 - val_pre_ner_accuracy: 0.8305\n",
      "Epoch 11/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 3.7370 - pre_intent_loss: 2.9522 - pre_ner_loss: 0.7848 - pre_intent_accuracy: 0.2263 - pre_ner_accuracy: 0.8455 - val_loss: 3.8006 - val_pre_intent_loss: 3.0243 - val_pre_ner_loss: 0.7763 - val_pre_intent_accuracy: 0.1562 - val_pre_ner_accuracy: 0.8370\n",
      "Epoch 12/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 3.6012 - pre_intent_loss: 2.8349 - pre_ner_loss: 0.7663 - pre_intent_accuracy: 0.2183 - pre_ner_accuracy: 0.8487 - val_loss: 3.6819 - val_pre_intent_loss: 2.9183 - val_pre_ner_loss: 0.7636 - val_pre_intent_accuracy: 0.1615 - val_pre_ner_accuracy: 0.8451\n",
      "Epoch 13/50\n",
      "35/35 [==============================] - 0s 7ms/step - loss: 3.4534 - pre_intent_loss: 2.7015 - pre_ner_loss: 0.7519 - pre_intent_accuracy: 0.2170 - pre_ner_accuracy: 0.8485 - val_loss: 3.5858 - val_pre_intent_loss: 2.8277 - val_pre_ner_loss: 0.7581 - val_pre_intent_accuracy: 0.1615 - val_pre_ner_accuracy: 0.8495\n",
      "Epoch 14/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 3.3097 - pre_intent_loss: 2.5697 - pre_ner_loss: 0.7400 - pre_intent_accuracy: 0.2143 - pre_ner_accuracy: 0.8485 - val_loss: 3.5048 - val_pre_intent_loss: 2.7549 - val_pre_ner_loss: 0.7498 - val_pre_intent_accuracy: 0.1615 - val_pre_ner_accuracy: 0.8508\n",
      "Epoch 15/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 3.1674 - pre_intent_loss: 2.4379 - pre_ner_loss: 0.7295 - pre_intent_accuracy: 0.2152 - pre_ner_accuracy: 0.8497 - val_loss: 3.4934 - val_pre_intent_loss: 2.7424 - val_pre_ner_loss: 0.7510 - val_pre_intent_accuracy: 0.1615 - val_pre_ner_accuracy: 0.8443\n",
      "Epoch 16/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 3.0609 - pre_intent_loss: 2.3485 - pre_ner_loss: 0.7124 - pre_intent_accuracy: 0.2129 - pre_ner_accuracy: 0.8562 - val_loss: 3.5140 - val_pre_intent_loss: 2.7308 - val_pre_ner_loss: 0.7831 - val_pre_intent_accuracy: 0.1823 - val_pre_ner_accuracy: 0.8352\n",
      "Epoch 17/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 3.0202 - pre_intent_loss: 2.3132 - pre_ner_loss: 0.7069 - pre_intent_accuracy: 0.2129 - pre_ner_accuracy: 0.8626 - val_loss: 3.4906 - val_pre_intent_loss: 2.7102 - val_pre_ner_loss: 0.7804 - val_pre_intent_accuracy: 0.2344 - val_pre_ner_accuracy: 0.8326\n",
      "Epoch 18/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 3.0906 - pre_intent_loss: 2.3603 - pre_ner_loss: 0.7303 - pre_intent_accuracy: 0.2138 - pre_ner_accuracy: 0.8494 - val_loss: 3.2511 - val_pre_intent_loss: 2.5256 - val_pre_ner_loss: 0.7256 - val_pre_intent_accuracy: 0.2083 - val_pre_ner_accuracy: 0.8419\n",
      "Epoch 19/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 3.0039 - pre_intent_loss: 2.3026 - pre_ner_loss: 0.7013 - pre_intent_accuracy: 0.2375 - pre_ner_accuracy: 0.8483 - val_loss: 3.1539 - val_pre_intent_loss: 2.4515 - val_pre_ner_loss: 0.7024 - val_pre_intent_accuracy: 0.2604 - val_pre_ner_accuracy: 0.8573\n",
      "Epoch 20/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 2.9213 - pre_intent_loss: 2.2328 - pre_ner_loss: 0.6884 - pre_intent_accuracy: 0.2321 - pre_ner_accuracy: 0.8523 - val_loss: 3.0150 - val_pre_intent_loss: 2.2820 - val_pre_ner_loss: 0.7331 - val_pre_intent_accuracy: 0.2812 - val_pre_ner_accuracy: 0.8562\n",
      "Epoch 21/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 2.7950 - pre_intent_loss: 2.1036 - pre_ner_loss: 0.6914 - pre_intent_accuracy: 0.3728 - pre_ner_accuracy: 0.8487 - val_loss: 2.9180 - val_pre_intent_loss: 2.1997 - val_pre_ner_loss: 0.7183 - val_pre_intent_accuracy: 0.1927 - val_pre_ner_accuracy: 0.8612\n",
      "Epoch 22/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 2.5733 - pre_intent_loss: 1.9090 - pre_ner_loss: 0.6643 - pre_intent_accuracy: 0.3603 - pre_ner_accuracy: 0.8615 - val_loss: 2.5772 - val_pre_intent_loss: 1.9899 - val_pre_ner_loss: 0.5873 - val_pre_intent_accuracy: 0.3802 - val_pre_ner_accuracy: 0.8758\n",
      "Epoch 23/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 2.3761 - pre_intent_loss: 1.7520 - pre_ner_loss: 0.6241 - pre_intent_accuracy: 0.4763 - pre_ner_accuracy: 0.8615 - val_loss: 2.5084 - val_pre_intent_loss: 1.8684 - val_pre_ner_loss: 0.6400 - val_pre_intent_accuracy: 0.5469 - val_pre_ner_accuracy: 0.8792\n",
      "Epoch 24/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 2.2292 - pre_intent_loss: 1.6171 - pre_ner_loss: 0.6121 - pre_intent_accuracy: 0.5638 - pre_ner_accuracy: 0.8649 - val_loss: 2.4288 - val_pre_intent_loss: 1.7959 - val_pre_ner_loss: 0.6330 - val_pre_intent_accuracy: 0.5417 - val_pre_ner_accuracy: 0.8781\n",
      "Epoch 25/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 2.1316 - pre_intent_loss: 1.5321 - pre_ner_loss: 0.5994 - pre_intent_accuracy: 0.5790 - pre_ner_accuracy: 0.8735 - val_loss: 2.3224 - val_pre_intent_loss: 1.7455 - val_pre_ner_loss: 0.5769 - val_pre_intent_accuracy: 0.5208 - val_pre_ner_accuracy: 0.8755\n",
      "Epoch 26/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 2.0883 - pre_intent_loss: 1.4856 - pre_ner_loss: 0.6027 - pre_intent_accuracy: 0.6455 - pre_ner_accuracy: 0.8664 - val_loss: 2.2133 - val_pre_intent_loss: 1.6582 - val_pre_ner_loss: 0.5552 - val_pre_intent_accuracy: 0.5938 - val_pre_ner_accuracy: 0.8737\n",
      "Epoch 27/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.9347 - pre_intent_loss: 1.3634 - pre_ner_loss: 0.5712 - pre_intent_accuracy: 0.6906 - pre_ner_accuracy: 0.8682 - val_loss: 2.0639 - val_pre_intent_loss: 1.5229 - val_pre_ner_loss: 0.5410 - val_pre_intent_accuracy: 0.6458 - val_pre_ner_accuracy: 0.8708\n",
      "Epoch 28/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.8709 - pre_intent_loss: 1.3097 - pre_ner_loss: 0.5612 - pre_intent_accuracy: 0.7321 - pre_ner_accuracy: 0.8715 - val_loss: 2.0309 - val_pre_intent_loss: 1.4714 - val_pre_ner_loss: 0.5595 - val_pre_intent_accuracy: 0.6927 - val_pre_ner_accuracy: 0.8805\n",
      "Epoch 29/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.8151 - pre_intent_loss: 1.2621 - pre_ner_loss: 0.5530 - pre_intent_accuracy: 0.7478 - pre_ner_accuracy: 0.8704 - val_loss: 2.0770 - val_pre_intent_loss: 1.4968 - val_pre_ner_loss: 0.5802 - val_pre_intent_accuracy: 0.6615 - val_pre_ner_accuracy: 0.8844\n",
      "Epoch 30/50\n",
      "35/35 [==============================] - 0s 6ms/step - loss: 1.7203 - pre_intent_loss: 1.1756 - pre_ner_loss: 0.5447 - pre_intent_accuracy: 0.7647 - pre_ner_accuracy: 0.8725 - val_loss: 1.9694 - val_pre_intent_loss: 1.4526 - val_pre_ner_loss: 0.5168 - val_pre_intent_accuracy: 0.6823 - val_pre_ner_accuracy: 0.8820\n",
      "Epoch 31/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.6876 - pre_intent_loss: 1.1396 - pre_ner_loss: 0.5480 - pre_intent_accuracy: 0.7790 - pre_ner_accuracy: 0.8740 - val_loss: 1.8342 - val_pre_intent_loss: 1.3134 - val_pre_ner_loss: 0.5208 - val_pre_intent_accuracy: 0.7500 - val_pre_ner_accuracy: 0.8771\n",
      "Epoch 32/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.6245 - pre_intent_loss: 1.0987 - pre_ner_loss: 0.5258 - pre_intent_accuracy: 0.7893 - pre_ner_accuracy: 0.8773 - val_loss: 1.7801 - val_pre_intent_loss: 1.2797 - val_pre_ner_loss: 0.5004 - val_pre_intent_accuracy: 0.6771 - val_pre_ner_accuracy: 0.8818\n",
      "Epoch 33/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.5575 - pre_intent_loss: 1.0476 - pre_ner_loss: 0.5099 - pre_intent_accuracy: 0.8076 - pre_ner_accuracy: 0.8780 - val_loss: 1.7569 - val_pre_intent_loss: 1.2242 - val_pre_ner_loss: 0.5327 - val_pre_intent_accuracy: 0.7500 - val_pre_ner_accuracy: 0.8870\n",
      "Epoch 34/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.5013 - pre_intent_loss: 0.9907 - pre_ner_loss: 0.5105 - pre_intent_accuracy: 0.8228 - pre_ner_accuracy: 0.8754 - val_loss: 1.7308 - val_pre_intent_loss: 1.2045 - val_pre_ner_loss: 0.5263 - val_pre_intent_accuracy: 0.8073 - val_pre_ner_accuracy: 0.8906\n",
      "Epoch 35/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.4389 - pre_intent_loss: 0.9376 - pre_ner_loss: 0.5014 - pre_intent_accuracy: 0.8246 - pre_ner_accuracy: 0.8792 - val_loss: 1.6355 - val_pre_intent_loss: 1.1524 - val_pre_ner_loss: 0.4831 - val_pre_intent_accuracy: 0.7812 - val_pre_ner_accuracy: 0.8875\n",
      "Epoch 36/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.3816 - pre_intent_loss: 0.8888 - pre_ner_loss: 0.4928 - pre_intent_accuracy: 0.8455 - pre_ner_accuracy: 0.8821 - val_loss: 1.6248 - val_pre_intent_loss: 1.1398 - val_pre_ner_loss: 0.4850 - val_pre_intent_accuracy: 0.7812 - val_pre_ner_accuracy: 0.8813\n",
      "Epoch 37/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.3667 - pre_intent_loss: 0.8738 - pre_ner_loss: 0.4929 - pre_intent_accuracy: 0.8652 - pre_ner_accuracy: 0.8790 - val_loss: 1.5759 - val_pre_intent_loss: 1.1044 - val_pre_ner_loss: 0.4715 - val_pre_intent_accuracy: 0.8021 - val_pre_ner_accuracy: 0.8909\n",
      "Epoch 38/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.3446 - pre_intent_loss: 0.8561 - pre_ner_loss: 0.4884 - pre_intent_accuracy: 0.8496 - pre_ner_accuracy: 0.8785 - val_loss: 1.6810 - val_pre_intent_loss: 1.2163 - val_pre_ner_loss: 0.4646 - val_pre_intent_accuracy: 0.6927 - val_pre_ner_accuracy: 0.8906\n",
      "Epoch 39/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.3413 - pre_intent_loss: 0.8650 - pre_ner_loss: 0.4763 - pre_intent_accuracy: 0.8607 - pre_ner_accuracy: 0.8800 - val_loss: 1.5334 - val_pre_intent_loss: 1.0595 - val_pre_ner_loss: 0.4739 - val_pre_intent_accuracy: 0.7188 - val_pre_ner_accuracy: 0.8904\n",
      "Epoch 40/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.2447 - pre_intent_loss: 0.7773 - pre_ner_loss: 0.4674 - pre_intent_accuracy: 0.9080 - pre_ner_accuracy: 0.8850 - val_loss: 1.4043 - val_pre_intent_loss: 0.9570 - val_pre_ner_loss: 0.4473 - val_pre_intent_accuracy: 0.8281 - val_pre_ner_accuracy: 0.8906\n",
      "Epoch 41/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.2098 - pre_intent_loss: 0.7544 - pre_ner_loss: 0.4554 - pre_intent_accuracy: 0.9116 - pre_ner_accuracy: 0.8864 - val_loss: 1.4223 - val_pre_intent_loss: 0.9591 - val_pre_ner_loss: 0.4632 - val_pre_intent_accuracy: 0.8229 - val_pre_ner_accuracy: 0.8932\n",
      "Epoch 42/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.1803 - pre_intent_loss: 0.7258 - pre_ner_loss: 0.4546 - pre_intent_accuracy: 0.9196 - pre_ner_accuracy: 0.8865 - val_loss: 1.3862 - val_pre_intent_loss: 0.9242 - val_pre_ner_loss: 0.4621 - val_pre_intent_accuracy: 0.8229 - val_pre_ner_accuracy: 0.8971\n",
      "Epoch 43/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.1335 - pre_intent_loss: 0.6832 - pre_ner_loss: 0.4503 - pre_intent_accuracy: 0.9277 - pre_ner_accuracy: 0.8863 - val_loss: 1.3070 - val_pre_intent_loss: 0.8736 - val_pre_ner_loss: 0.4334 - val_pre_intent_accuracy: 0.8229 - val_pre_ner_accuracy: 0.8966\n",
      "Epoch 44/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.1122 - pre_intent_loss: 0.6699 - pre_ner_loss: 0.4423 - pre_intent_accuracy: 0.9196 - pre_ner_accuracy: 0.8871 - val_loss: 1.2868 - val_pre_intent_loss: 0.8654 - val_pre_ner_loss: 0.4214 - val_pre_intent_accuracy: 0.8542 - val_pre_ner_accuracy: 0.8966\n",
      "Epoch 45/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.0819 - pre_intent_loss: 0.6432 - pre_ner_loss: 0.4386 - pre_intent_accuracy: 0.9299 - pre_ner_accuracy: 0.8835 - val_loss: 1.2237 - val_pre_intent_loss: 0.8045 - val_pre_ner_loss: 0.4192 - val_pre_intent_accuracy: 0.8594 - val_pre_ner_accuracy: 0.8951\n",
      "Epoch 46/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.0251 - pre_intent_loss: 0.5942 - pre_ner_loss: 0.4310 - pre_intent_accuracy: 0.9384 - pre_ner_accuracy: 0.8887 - val_loss: 1.2017 - val_pre_intent_loss: 0.7902 - val_pre_ner_loss: 0.4115 - val_pre_intent_accuracy: 0.8750 - val_pre_ner_accuracy: 0.8964\n",
      "Epoch 47/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 1.0001 - pre_intent_loss: 0.5771 - pre_ner_loss: 0.4230 - pre_intent_accuracy: 0.9424 - pre_ner_accuracy: 0.8889 - val_loss: 1.1680 - val_pre_intent_loss: 0.7534 - val_pre_ner_loss: 0.4146 - val_pre_intent_accuracy: 0.8802 - val_pre_ner_accuracy: 0.8982\n",
      "Epoch 48/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 0.9758 - pre_intent_loss: 0.5549 - pre_ner_loss: 0.4209 - pre_intent_accuracy: 0.9433 - pre_ner_accuracy: 0.8887 - val_loss: 1.1476 - val_pre_intent_loss: 0.7358 - val_pre_ner_loss: 0.4117 - val_pre_intent_accuracy: 0.8750 - val_pre_ner_accuracy: 0.9031\n",
      "Epoch 49/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 0.9539 - pre_intent_loss: 0.5356 - pre_ner_loss: 0.4183 - pre_intent_accuracy: 0.9415 - pre_ner_accuracy: 0.8875 - val_loss: 1.1042 - val_pre_intent_loss: 0.7068 - val_pre_ner_loss: 0.3973 - val_pre_intent_accuracy: 0.8698 - val_pre_ner_accuracy: 0.8982\n",
      "Epoch 50/50\n",
      "35/35 [==============================] - 0s 5ms/step - loss: 0.9277 - pre_intent_loss: 0.5163 - pre_ner_loss: 0.4114 - pre_intent_accuracy: 0.9509 - pre_ner_accuracy: 0.8929 - val_loss: 1.1351 - val_pre_intent_loss: 0.7396 - val_pre_ner_loss: 0.3955 - val_pre_intent_accuracy: 0.8594 - val_pre_ner_accuracy: 0.8982\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7ff34c4fbe50>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_dataset,epochs=params['epochs'],validation_data=valid_dataset,callbacks=[checkpoint,reduce_lr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights('../ner_model_weight/model_encoder_714.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('../ner_model_weight/model_encoder_714.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pre_intent,pre_slot = model.predict([[111, 196, 182, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2.9494049e-05, 5.3530930e-05, 8.8762766e-04, 1.3331781e-03,\n",
       "        8.2310438e-03, 2.2651965e-04, 6.9630548e-02, 1.3630442e-02,\n",
       "        1.6755251e-03, 6.0249095e-05, 1.6953712e-04, 8.2841929e-04,\n",
       "        1.7529672e-03, 1.6962040e-04, 8.7594418e-03, 1.6872374e-03,\n",
       "        8.2052469e-02, 3.5339527e-04, 3.5789917e-04, 9.6443466e-05,\n",
       "        1.8358817e-02, 1.3664477e-03, 3.3313471e-03, 2.9194590e-03,\n",
       "        5.9483160e-04, 8.1444468e-04, 8.0008281e-04, 6.7297084e-04,\n",
       "        1.0471063e-03, 4.7507658e-04, 6.4713242e-03, 9.1811264e-05,\n",
       "        2.4078325e-03, 2.3891955e-05, 1.8400388e-03, 5.8634733e-03,\n",
       "        1.7386231e-04, 8.7405009e-05, 5.2923884e-04, 6.7850437e-05,\n",
       "        4.5236826e-04, 1.2026340e-03, 4.8980106e-05, 2.4951095e-04,\n",
       "        3.3015964e-04, 2.8099646e-04, 2.6102316e-01, 1.0366692e-03,\n",
       "        5.1147066e-04, 5.6982944e-03, 3.2021280e-04, 6.5410853e-04,\n",
       "        1.4881464e-04, 3.6891075e-03, 6.1456543e-05]], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_intent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[1.41103761e-02, 1.25193343e-04, 5.41495217e-04, 9.07725375e-03,\n",
       "         6.08455914e-04, 3.31322923e-02, 1.71928643e-03, 1.30036788e-04,\n",
       "         2.08800237e-04, 1.61213626e-03, 3.38990561e-04, 5.23552124e-04,\n",
       "         1.68756989e-03, 1.57017086e-04, 2.48261523e-02, 8.47355742e-03,\n",
       "         1.30898831e-03, 2.78503197e-04, 2.69482918e-02, 3.11168318e-04,\n",
       "         3.42854677e-04, 2.83676032e-02, 5.70192351e-04, 1.13033201e-03,\n",
       "         8.25727533e-04, 3.43502918e-03, 7.18734682e-01, 2.02587224e-03,\n",
       "         5.22225630e-04, 1.63231033e-03, 1.07515727e-04, 1.60128140e-04,\n",
       "         1.52094217e-04, 5.78077976e-04, 1.70798413e-02, 4.30360157e-03],\n",
       "        [9.36264638e-03, 2.61177862e-04, 5.07572899e-04, 6.82142470e-03,\n",
       "         1.46548473e-03, 3.72575037e-02, 5.16129285e-03, 1.45172686e-04,\n",
       "         4.16871655e-04, 9.11060488e-05, 5.35520900e-04, 5.96660539e-04,\n",
       "         5.58356522e-03, 2.08530459e-04, 6.41897172e-02, 7.81682506e-03,\n",
       "         1.67384301e-03, 1.25431130e-03, 1.42141178e-01, 4.28864092e-04,\n",
       "         3.29936884e-04, 4.01344486e-02, 9.42111801e-05, 4.21895646e-03,\n",
       "         3.56626022e-03, 1.05874175e-02, 7.61181042e-02, 7.57726433e-04,\n",
       "         1.69313367e-04, 1.01893907e-03, 2.03479154e-04, 1.66374608e-04,\n",
       "         2.52803264e-04, 2.06224900e-03, 5.47106445e-01, 1.28160315e-02],\n",
       "        [3.94533761e-03, 2.18722867e-04, 1.00111996e-03, 5.96914114e-03,\n",
       "         1.08370080e-03, 1.22123938e-02, 6.77781703e-04, 4.92897234e-05,\n",
       "         4.14839560e-05, 8.43859161e-04, 1.00546647e-04, 5.91578064e-05,\n",
       "         1.86106109e-03, 2.08075435e-04, 4.18045163e-01, 4.07093205e-04,\n",
       "         2.52723490e-04, 8.08755809e-04, 1.92638533e-03, 1.76148897e-04,\n",
       "         1.32926172e-04, 1.43287648e-02, 9.24679276e-04, 1.02151753e-02,\n",
       "         4.26952873e-04, 1.59909338e-04, 3.80601618e-03, 2.81920307e-03,\n",
       "         1.34576359e-04, 4.83951211e-04, 5.32288614e-05, 9.13928452e-05,\n",
       "         8.92181881e-04, 5.88117726e-03, 2.46600923e-03, 5.04205981e-03],\n",
       "        [4.27365070e-03, 7.66324156e-05, 4.61868389e-04, 1.10336738e-02,\n",
       "         5.60781336e-04, 5.41674614e-01, 7.25032296e-04, 1.16693678e-04,\n",
       "         2.17722874e-04, 8.48549884e-04, 5.22752991e-04, 3.59956379e-04,\n",
       "         7.00030359e-04, 2.27197772e-04, 1.30035514e-02, 4.65742196e-04,\n",
       "         4.32271976e-04, 3.39259073e-04, 1.92930480e-03, 1.84619887e-04,\n",
       "         2.16623550e-04, 1.33729041e-01, 4.97064437e-04, 8.95820500e-04,\n",
       "         3.09996627e-04, 1.72849582e-03, 1.38668597e-01, 2.11483077e-03,\n",
       "         6.39105274e-04, 7.94358377e-04, 1.71103966e-04, 8.55431135e-05,\n",
       "         1.27472973e-04, 3.21901264e-03, 3.73321329e-03, 2.14039371e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03],\n",
       "        [6.57334685e-01, 2.82676174e-05, 4.84353077e-05, 8.58868181e-04,\n",
       "         1.06622370e-04, 1.25480089e-02, 6.48852001e-05, 1.91132167e-05,\n",
       "         4.85082019e-05, 6.60983933e-05, 9.45026113e-05, 8.53004458e-05,\n",
       "         1.67672246e-04, 3.38526988e-05, 1.30034483e-03, 9.41612452e-05,\n",
       "         2.56531384e-05, 2.23298630e-05, 4.85386379e-04, 1.82623589e-05,\n",
       "         5.01784525e-05, 1.49929291e-03, 6.39891223e-05, 1.83195138e-04,\n",
       "         6.18731719e-05, 5.00940951e-04, 4.91311168e-03, 3.47057328e-04,\n",
       "         5.17839399e-05, 1.40643635e-04, 7.12745750e-05, 2.37499298e-05,\n",
       "         2.59950830e-05, 4.20225144e-04, 9.03238077e-04, 1.44769333e-03]]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_slot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
